{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#本章需导入的模块\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.rcParams['font.sans-serif']=['SimHei']  #解决中文显示乱码问题\n",
    "plt.rcParams['axes.unicode_minus']=False\n",
    "import warnings\n",
    "warnings.filterwarnings(action = 'ignore')\n",
    "from scipy.stats import beta\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "import sklearn.linear_model as LM\n",
    "from sklearn.model_selection import cross_val_score,cross_validate,train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import roc_curve, auc,accuracy_score,precision_recall_curve\n",
    "#本节增加导入的模块\n",
    "import jieba\n",
    "import jieba.analyse\n",
    "from sklearn.feature_extraction.text import  TfidfVectorizer\n",
    "import json\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\xuewe\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.709 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文本分词结果：\n",
      " ['中国 的 发展 是 开放 的 发展', '中国 经济 发展 的 质量 在 稳步 提升 ， 人民 生活 在 持续 改善', '从 集市 、 超市 到 网购 ， 线上 年货 成为 中国 老百姓 最 便捷 的 硬核 年货', '支付 体验 的 优化 以及 物流配送 效率 的 提升 ， 线上 购物 变得 越来越 便利']\n",
      "特征词表：\n",
      " ['中国', '人民', '以及', '优化', '体验', '便利', '便捷', '发展', '变得', '年货', '开放', '成为', '持续', '提升', '支付', '改善', '效率', '物流配送', '生活', '硬核', '稳步', '线上', '经济', '网购', '老百姓', '质量', '购物', '超市', '越来越', '集市']\n",
      "idf:\n",
      " [1.22314355 1.91629073 1.91629073 1.91629073 1.91629073 1.91629073\n",
      " 1.91629073 1.51082562 1.91629073 1.91629073 1.91629073 1.91629073\n",
      " 1.91629073 1.51082562 1.91629073 1.91629073 1.91629073 1.91629073\n",
      " 1.91629073 1.91629073 1.91629073 1.51082562 1.91629073 1.91629073\n",
      " 1.91629073 1.91629073 1.91629073 1.91629073 1.91629073 1.91629073]\n",
      "中国 0.32346721385745636\n",
      "人民 0.0\n",
      "以及 0.0\n",
      "优化 0.0\n",
      "体验 0.0\n",
      "便利 0.0\n",
      "便捷 0.0\n",
      "发展 0.7990927223856119\n",
      "变得 0.0\n",
      "年货 0.0\n",
      "开放 0.5067738969102946\n",
      "成为 0.0\n",
      "持续 0.0\n",
      "提升 0.0\n",
      "支付 0.0\n",
      "改善 0.0\n",
      "效率 0.0\n",
      "物流配送 0.0\n",
      "生活 0.0\n",
      "硬核 0.0\n",
      "稳步 0.0\n",
      "线上 0.0\n",
      "经济 0.0\n",
      "网购 0.0\n",
      "老百姓 0.0\n",
      "质量 0.0\n",
      "购物 0.0\n",
      "超市 0.0\n",
      "越来越 0.0\n",
      "集市 0.0\n",
      "中国 0.21701663412516095\n",
      "人民 0.3399984933818611\n",
      "以及 0.0\n",
      "优化 0.0\n",
      "体验 0.0\n",
      "便利 0.0\n",
      "便捷 0.0\n",
      "发展 0.2680587174477245\n",
      "变得 0.0\n",
      "年货 0.0\n",
      "开放 0.0\n",
      "成为 0.0\n",
      "持续 0.3399984933818611\n",
      "提升 0.2680587174477245\n",
      "支付 0.0\n",
      "改善 0.3399984933818611\n",
      "效率 0.0\n",
      "物流配送 0.0\n",
      "生活 0.3399984933818611\n",
      "硬核 0.0\n",
      "稳步 0.3399984933818611\n",
      "线上 0.0\n",
      "经济 0.3399984933818611\n",
      "网购 0.0\n",
      "老百姓 0.0\n",
      "质量 0.3399984933818611\n",
      "购物 0.0\n",
      "超市 0.0\n",
      "越来越 0.0\n",
      "集市 0.0\n",
      "中国 0.18403533402582276\n",
      "人民 0.0\n",
      "以及 0.0\n",
      "优化 0.0\n",
      "体验 0.0\n",
      "便利 0.0\n",
      "便捷 0.2883269135107866\n",
      "发展 0.0\n",
      "变得 0.0\n",
      "年货 0.5766538270215732\n",
      "开放 0.0\n",
      "成为 0.2883269135107866\n",
      "持续 0.0\n",
      "提升 0.0\n",
      "支付 0.0\n",
      "改善 0.0\n",
      "效率 0.0\n",
      "物流配送 0.0\n",
      "生活 0.0\n",
      "硬核 0.2883269135107866\n",
      "稳步 0.0\n",
      "线上 0.22732025037110293\n",
      "经济 0.0\n",
      "网购 0.2883269135107866\n",
      "老百姓 0.2883269135107866\n",
      "质量 0.0\n",
      "购物 0.0\n",
      "超市 0.2883269135107866\n",
      "越来越 0.0\n",
      "集市 0.2883269135107866\n",
      "中国 0.0\n",
      "人民 0.0\n",
      "以及 0.2982327375202219\n",
      "优化 0.2982327375202219\n",
      "体验 0.2982327375202219\n",
      "便利 0.2982327375202219\n",
      "便捷 0.0\n",
      "发展 0.0\n",
      "变得 0.2982327375202219\n",
      "年货 0.0\n",
      "开放 0.0\n",
      "成为 0.0\n",
      "持续 0.0\n",
      "提升 0.2351301157996824\n",
      "支付 0.2982327375202219\n",
      "改善 0.0\n",
      "效率 0.2982327375202219\n",
      "物流配送 0.2982327375202219\n",
      "生活 0.0\n",
      "硬核 0.0\n",
      "稳步 0.0\n",
      "线上 0.2351301157996824\n",
      "经济 0.0\n",
      "网购 0.0\n",
      "老百姓 0.0\n",
      "质量 0.0\n",
      "购物 0.2982327375202219\n",
      "超市 0.0\n",
      "越来越 0.2982327375202219\n",
      "集市 0.0\n"
     ]
    }
   ],
   "source": [
    "documents = [\"中国的发展是开放的发展\",\n",
    "    \"中国经济发展的质量在稳步提升，人民生活在持续改善\",\n",
    "    \"从集市、超市到网购，线上年货成为中国老百姓最便捷的硬核年货\",\n",
    "    \"支付体验的优化以及物流配送效率的提升，线上购物变得越来越便利\"]\n",
    "documents = [\" \".join(jieba.cut(item)) for item in documents]\n",
    "print(\"文本分词结果：\\n\",documents)\n",
    "vectorizer = TfidfVectorizer()  #定义TF-IDF对象\n",
    "X = vectorizer.fit_transform(documents)\n",
    "\n",
    "words=vectorizer.get_feature_names()\n",
    "print(\"特征词表：\\n\",words)  \n",
    "print(\"idf:\\n\",vectorizer.idf_)  #idf\n",
    "X=X.toarray() #print(X.toarray())   #文本-词的tf-idf矩阵\n",
    "for i in range(len(X)): ##打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重 \n",
    "    for j in range(len(words)):\n",
    "        print(words[j],X[i][j])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "说明：这里以一个简单示例说明文本分类研究的数据预处理过程。 \n",
    "1、这里列表documents中的四个元素分别存储四句文字。例如：第一句\"中国的发展是开放的发展\"，第二句\"中国经济发展的质量在稳步提升，人民生活在持续改善\"等。分别代表四篇文本。\n",
    "2、文本处理的第一步：将句子分割成若干个词，即分词。目前Python支持多种分词算法，其中jieba分词是使用较为普遍的。利用jieba.cut对每句话分词。 \n",
    "3、文本处理的第二步：文本量化。最基本的方式是对量化分词结果。这里采用词的TF-IDF计量。利用TfidfVectorizer()实现。\n",
    "4、文本处理的第三步：量化文本的重新组织。通常以词为行，文本为列。通过vectorizer.fit_transform()实现。由此可得到文本的特征词表和对应的idf等。\n",
    "5、为实现后续的文本分类，应将文本转成行为文本列为词的矩阵形式。通过toarray()实现。该矩阵存放的是每个特征词的TF-IDF权重，分别将作为输入变量给到模型。例如，“中国”在第一篇文本的权重为0.32，“发展”的权重为0.799等等。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "朴素贝叶斯分类器的训练误差:0.191\n",
      "朴素贝叶斯分类器的测试误差:0.521\n"
     ]
    }
   ],
   "source": [
    "alltext=[]\n",
    "label=[]\n",
    "fenceText=[]\n",
    "fn = open('离婚诉讼文本.json', 'r',encoding='utf-8')\n",
    "line = fn.readline()\n",
    "while line:\n",
    "    data = json.loads(line)\n",
    "    for sent in data:\n",
    "        if len(sent['labels']) ==1:\n",
    "            label.append(sent['labels'])\n",
    "            alltext.append(sent['sentence'])\n",
    "    line=fn.readline()\n",
    "fn.close()\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(alltext,label,train_size=0.60, random_state=123) \n",
    "fenceText=[\" \".join(jieba.cut(item)) for item in X_train]\n",
    "with open(\"停用词表.txt\", \"r\" ,encoding='utf-8') as fn:\n",
    "    stpwrdlst = fn.read().splitlines()\n",
    "fn.close()        \n",
    "vectorizer = TfidfVectorizer(stop_words=stpwrdlst,max_features=400)\n",
    "X_train = vectorizer.fit_transform(fenceText)\n",
    "X_train=X_train.toarray()\n",
    "modelNB = MultinomialNB()\n",
    "modelNB.fit(X_train, Y_train)\n",
    "print(\"朴素贝叶斯分类器的训练误差:%.3f\"%(1-modelNB.score(X_train,Y_train)))\n",
    "\n",
    "fenceText=[\" \".join(jieba.cut(item)) for item in X_test]\n",
    "X_test = vectorizer.fit_transform(fenceText)\n",
    "X_test=X_test.toarray()\n",
    "print(\"朴素贝叶斯分类器的测试误差:%.3f\"%(1-modelNB.score(X_test,Y_test)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "说明：这里以一个实际的文本分类问题为例，说明朴素贝叶斯分类器的应用\n",
    "1、引用json，并利用json.loads将json本文读入Python数据字典。这里首先涉及Python文件读取操作。一般应首先定义文件句柄（如这里的fn)，即告知以怎样的方式（读r或写w等）打开哪个文件，并指定中文采用utf-8编码方式（utf-8是目前使用最为广泛的一种万国码unicode实现方式）编码。其次，可利用readline()逐行读取文本内容。\n",
    "2、TfidfVectorizer(stop_words=stpwrdlst,max_features=400)，指定以\"停用词表.txt\"文件中的词作为停用词。仅提取前400个最重要（权重最大）的特征词。\n",
    "3、将文本数据随机分成训练集和测试集，比例为6：4。同时，为防止测试集信息“泄露”给模型，应分别进行训练文本和测试文本的文本量化。\n",
    "4、本例采用的是多项式朴素贝叶斯分类器（ MultinomialNB），适合基于计数的整型（离散型）输入变量。尽管这里的输入变量（tf-idf)是小数型（连续型），但也可将其视为基于词频计算的特殊的小数型计数。\n",
    "5、本例的贝叶斯分类器的训练误差和测试误差均较高。该问题涉及的方面较多相对复杂。一方面可优化分词和TF-IDF计算。例如补充法律专业用词，完善停用词表等。或者尝试其他分类模型。另一方面也说明采用tf-idf量化文本并非最佳方案。目前另一种较为流行的文本量化方式是采用基于word2vec的词向量。有兴趣的读者可参考相关资料。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
